package com.maistars.spider.service.capture;

import com.maistars.spider.common.constants.PageType;
import com.maistars.spider.common.constants.RenderEngineType;
import com.maistars.spider.common.exception.TopicSpiderException;
import com.maistars.spider.service.domain.capture.CapturedPage;
import com.maistars.spider.service.domain.rule.PageRule;
import com.maistars.spider.service.domain.render.RenderConfig;
import com.maistars.spider.service.domain.rule.SiteRule;
import com.maistars.spider.service.domain.task.CrawlerTaskInst;
import com.maistars.spider.service.parser.PageRuleParser;
import com.maistars.spider.service.domain.render.PageRequest;
import com.maistars.spider.service.domain.render.PageResponse;
import com.maistars.spider.service.parser.SiteRuleParser;
import com.maistars.spider.service.render.RenderService;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

/**
 * @author dhr
 * @date 2021/1/17 上午12:26
 */
public class CaptureEngine {

    private static final int TIMEOUT = 15000;

    private PageRuleParser pageRuleParser = new PageRuleParser();

    private SiteRuleParser siteRuleParser = new SiteRuleParser();

    private ExecutorService executorService;

    private CrawlerTaskInst crawlerTaskInst;

    private List<PageRule> pageRules;

    private SiteRule siteRule;

    private RenderService renderService = new RenderService();

    public CaptureEngine(CrawlerTaskInst crawlerTaskInst, SiteRule siteRule, List<PageRule> pageRules, int threadNum) {
        this.crawlerTaskInst = crawlerTaskInst;
        executorService = Executors.newFixedThreadPool(threadNum);
        this.pageRules = pageRules;
        this.siteRule = siteRule;
    }

    private List<CapturedPage> capturePage(String containerPageContent, PageRule pageRule) throws InterruptedException {
        // 读取页面规则
        List<String> realUrls = pageRuleParser.getRealUrls(containerPageContent, pageRule.getParentRule());
        if (realUrls == null || realUrls.isEmpty()) {
            return new ArrayList<>();
        }
        Map<String, CapturedPage> capturedPageMap = new HashMap<>();
        CountDownLatch countDownLatch = new CountDownLatch(realUrls.size());
        for (String realUrl : realUrls) {
            executorService.execute(new Runnable() {
                @Override
                public void run() {
                    try {
                        // 最后一个页面规则，不获取html内容
                        if (pageRules.size() == pageRule.getLevel() && PageType.CONTENT.equals(pageRule.getPageType())) {
                            CapturedPage capturedPage = renderPage(realUrl, pageRule.getRenderConfig());
                            capturedPageMap.put(realUrl, capturedPage);
                        } else {
                            // 需要获取html内容，目的是为了下个页面的url解析
                            CapturedPage capturedPage = renderPage(realUrl, pageRule.getRenderConfig());
                            // 分页url处理
                            List<String> pageRealUrls = pageRuleParser.getRealUrls(capturedPage.getPageContent(), pageRule.getPageTurnRule());
                            for (String pageRealUrl : pageRealUrls) {
                                CapturedPage pageCapturedPage = renderPage(pageRealUrl, pageRule.getRenderConfig());
                                capturedPage.getTurnCapturedPages().add(pageCapturedPage);
                            }
                            capturedPageMap.put(realUrl, capturedPage);
                        }
                    } catch (TopicSpiderException e) {
                        e.printStackTrace();
                    } finally {
                        countDownLatch.countDown();
                    }
                }
            });
        }
        countDownLatch.await();
        List<CapturedPage> capturedPages = new ArrayList<>(capturedPageMap.values());
        return capturedPages;
    }

    private CapturedPage renderPage(String realUrl, RenderConfig renderConfig) throws TopicSpiderException {
        PageRequest pageRequest = new PageRequest();
        pageRequest.setUrl(realUrl);
        pageRequest.setRenderConfig(renderConfig);

        PageResponse pageResponse = renderService.renderPage(RenderEngineType.SIMPLE, pageRequest);
        CapturedPage capturedPage = new CapturedPage();
        capturedPage.setStatus(pageResponse.getStatus());
        capturedPage.setCurrentUrl(pageResponse.getUrl());
        capturedPage.setPageContent(pageResponse.getRenderResult());
        return capturedPage;
    }

    public Map<Integer,List<CapturedPage>> execute() throws TopicSpiderException {
        Map<Integer,List<CapturedPage>> capturedPageLevels = new HashMap<>();
        try {
            if (pageRules == null || pageRules.isEmpty()) {
                return null;
            }
            // 处理网站入口网页规则
            List<String> siteUrls = siteRuleParser.getRealUrls(siteRule);

            // 写入其他子级的规则
            for (int i = 1; i <= pageRules.size(); i++) {
                if (i != 1) {
                    // 那上一级规则捞取当前层级数据
                    capturedPageLevels.put(i, new ArrayList<>());
                    for (CapturedPage capturedPage : capturedPageLevels.get(i - 1)) {
                        if (capturedPage.getStatus() != 0) {
                            continue;
                        }
                        List<CapturedPage> capturedPages = capturePage(capturedPage.getPageContent(), pageRules.get(i-1));
                        if(crawlerTaskInst.getCapturedPageTotoalCount().get(i) == null){
                            crawlerTaskInst.getCapturedPageTotoalCount().put(i, new AtomicInteger(0));
                        }
                        for (CapturedPage hasCapturedPage : capturedPages) {
                            if (hasCapturedPage.getStatus() == 0) {
                                if(crawlerTaskInst.getCapturedPageSuccessCount().get(i) == null) {
                                    crawlerTaskInst.getCapturedPageSuccessCount().put(i, new AtomicInteger(0));
                                }
                                crawlerTaskInst.getCapturedPageSuccessCount().get(i).getAndAdd(1);
                                if (capturedPageLevels.get(i) == null) {
                                    capturedPageLevels.put(i, new ArrayList<>());
                                }
                                capturedPageLevels.get(i).add(hasCapturedPage);
                            }
                        }
                        crawlerTaskInst.getCapturedPageTotoalCount().get(i).getAndAdd(1);
                    }
                } else {
                    // 写入列名
                    crawlerTaskInst.getTaskInstResultStore()
                            .getCapturedPageStore().writeColumnNames();
                    for(String siteUrl:siteUrls) {
                        // 先获取入口规则的网页内容
                        CapturedPage startCapturedPage = renderPage(siteUrl, pageRules.get(i-1).getRenderConfig());
                        if(crawlerTaskInst.getCapturedPageTotoalCount().get(i) == null){
                            crawlerTaskInst.getCapturedPageTotoalCount().put(i, new AtomicInteger(0));
                        }
                        if (startCapturedPage.getStatus() == 0) {
                            if(crawlerTaskInst.getCapturedPageSuccessCount().get(i) == null) {
                                crawlerTaskInst.getCapturedPageSuccessCount().put(i, new AtomicInteger(0));
                            }
                            crawlerTaskInst.getCapturedPageSuccessCount().get(i).getAndAdd(1);
                            if (capturedPageLevels.get(i) == null) {
                                capturedPageLevels.put(i, new ArrayList<>());
                            }
                            capturedPageLevels.get(i).add(startCapturedPage);
                        }
                        crawlerTaskInst.getCapturedPageTotoalCount().get(i).getAndAdd(1);
                    }
                }
                List<String> records = capturedPageLevels.get(i).stream().map(capturedPage1 -> capturedPage1.getCurrentUrl()).collect(Collectors.toList());
                crawlerTaskInst.getTaskInstResultStore()
                        .getCapturedPageStore().writeSuccessUrls(i, records);
            }
        } catch (InterruptedException e) {
            throw new TopicSpiderException("capture page has errors", e);
        }
        return capturedPageLevels;
    }
}
